knitr::opts_chunk$set(echo=TRUE, message=FALSE, warning=FALSE, dpi=60, out.width = "100%", eval = TRUE)
options(scipen=999)
options(knitr.kable.NA = '--') #'--'
options(knitr.kable.NAN = '--')


# load our package
library(reticulate)

Make an Issue

Let’s make an issue to document what we are doing. We turn this chunk off because it is a one time call.

output_issue_create <- gh::gh(
  endpoint = "POST /repos/{owner}/{repo}/issues",
  title = "`mergin-client` error when changes made to geopackages",
  body = "Seems to be occurring with programatic and `QGIS` by-hand changes to files.",
  owner = params$repo_owner,
  repo = params$repo_name,
  assignee = params$repo_owner
)

# lets add labels
gh::gh(
  endpoint = "POST /repos/{owner}/{repo}/issues/{issue_number}/labels",
  labels = c("QGIS", "mergin", "documentation"),
  owner = params$repo_owner,
  repo = params$repo_name,
  issue_number = output_issue_create$number
)

output_issue_create$number

Here is the issue number though captured from above:

output_issue_create$number

[1] 166

Create Fresh QGIS Project

Ok. This is a brand new project!

I created it on the command line and pushed it to the server with:

mergin create newgraph/py_client_error --from-dir ~/Projects/gis/py_client_error

Then we want to start fresh - delete the project locally and then download to our local machine. We could use system2 calls to our already existing conda environment like below which seems to work just fine as so…

# first we remove the file
system2("rm", args = c("-rf", "/Users/airvine/Projects/gis/py_client_error"))

# now we download
system2('conda', args = c('run', '-n', 'dff2', 'mergin download newgraph/py_client_error /Users/airvine/Projects/gis/py_client_error'), stdout = TRUE)
## [1] "Downloading into /Users/airvine/Projects/gis/py_client_error"
## [2] ""                                                            
## [3] "Done"                                                        
## [4] ""

Use reticulate to install python and set up virtual environment

But… We want to move to using python directly in Rstudio (helps for story telling and reproducability) with reticulate So we installed python fresh with:

reticulate::install_python()

And now:

reticulate::virtualenv_starter(all = TRUE)
##   version                                                 path
## 1  3.12.3 /Users/airvine/.pyenv/versions/3.12.3/bin/python3.12

Install mergin-client package in the virtual environment

So do a one time move and make a py_env called test and install the mergin-client package

reticulate::virtualenv_create("test")

reticulate::virtualenv_install("test", "mergin-client")

Now lets indicate we want to use it

reticulate::use_virtualenv("test", required = TRUE)

Import mergin module and create a client instance

  1. Import the mergin module:
mergin <- import("mergin")
  1. Create an instance of mergin-client with the specified login credentials.
# Create a Mergin client instance with your credentials
client <- mergin$MerginClient(login = Sys.getenv("MERGIN_USERNAME"), password = Sys.getenv("MERGIN_PASSWORD"))
  1. Remove the project and create it again
system2("rm", args = c("-rf", "/Users/airvine/Projects/gis/py_client_error"))


# Download a project using the client instance
client$download_project('newgraph/py_client_error', '/Users/airvine/Projects/gis/py_client_error')

Review the methods and attributes of the module and client instance

Whoa - awesome. Let’s have a look at all the available

# List attributes and methods of the module or client instance
module_methods <- py_list_attributes(mergin)
client_methods <- py_list_attributes(client)

# Print the lists
print("Methods and attributes of the module:")
## [1] "Methods and attributes of the module:"
print(module_methods)
##  [1] "ClientError"    "InvalidProject" "LoginError"     "MerginClient"   "MerginProject" 
##  [6] "__builtins__"   "__cached__"     "__doc__"        "__file__"       "__loader__"    
## [11] "__name__"       "__package__"    "__path__"       "__spec__"       "__version__"   
## [16] "client"         "client_pull"    "client_push"    "common"         "merginproject" 
## [21] "utils"          "version"
print("Methods and attributes of the client instance:")
## [1] "Methods and attributes of the client instance:"
print(client_methods)
##  [1] "__class__"                            "__delattr__"                         
##  [3] "__dict__"                             "__dir__"                             
##  [5] "__doc__"                              "__eq__"                              
##  [7] "__format__"                           "__ge__"                              
##  [9] "__getattribute__"                     "__getstate__"                        
## [11] "__gt__"                               "__hash__"                            
## [13] "__init__"                             "__init_subclass__"                   
## [15] "__le__"                               "__lt__"                              
## [17] "__module__"                           "__ne__"                              
## [19] "__new__"                              "__reduce__"                          
## [21] "__reduce_ex__"                        "__repr__"                            
## [23] "__setattr__"                          "__sizeof__"                          
## [25] "__str__"                              "__subclasshook__"                    
## [27] "__weakref__"                          "_auth_params"                        
## [29] "_auth_session"                        "_check_token"                        
## [31] "_do_request"                          "_server_type"                        
## [33] "_server_version"                      "_user_info"                          
## [35] "add_user_permissions_to_project"      "client_version"                      
## [37] "clone_project"                        "create_project"                      
## [39] "create_project_and_push"              "create_workspace"                    
## [41] "default_url"                          "delete_project"                      
## [43] "delete_project_now"                   "download_file"                       
## [45] "download_file_diffs"                  "download_files"                      
## [47] "download_project"                     "get"                                 
## [49] "get_file_diff"                        "get_projects_by_names"               
## [51] "has_unfinished_pull"                  "has_writing_permissions"             
## [53] "is_server_compatible"                 "log"                                 
## [55] "login"                                "opener"                              
## [57] "paginated_projects_list"              "post"                                
## [59] "project_file_changeset_info"          "project_file_history_info"           
## [61] "project_info"                         "project_status"                      
## [63] "project_user_permissions"             "project_version_info"                
## [65] "project_versions"                     "projects_list"                       
## [67] "pull_project"                         "push_project"                        
## [69] "remove_user_permissions_from_project" "rename_project"                      
## [71] "reset_local_changes"                  "resolve_unfinished_pull"             
## [73] "server_type"                          "server_version"                      
## [75] "set_project_access"                   "setup_logging"                       
## [77] "url"                                  "user_agent_info"                     
## [79] "user_info"                            "user_service"                        
## [81] "username"                             "workspace_service"                   
## [83] "workspaces_list"

Sick.

Get help on a specific method. Opens in new window which is a bit odd IMO. We will turn this chunk off to avoid clicking the closed.

# Get help on a specific method
py_help(client$download_project)
py_help(client$project_info)
py_help(client$push_project)

Let’s get some info about the project.

project_info <- client$project_info('newgraph/py_client_error')

Lot’s here. for instance

names(project_info)
##  [1] "access"       "created"      "creator"      "disk_usage"   "files"        "id"          
##  [7] "name"         "namespace"    "permissions"  "removed_at"   "role"         "tags"        
## [13] "updated"      "uploads"      "version"      "workspace_id"

Use processx package and mergin-client commandline tool and current conda environment to check the status

mergin status is not an obvious call with the python module raw so we will use the mergin-client installed with conda. We will probably move away from conda soon and straight to venv so not getting into the conda specifics. We do havetwo conda environments installed (latest one yesterday) and they are created using the environment.yml files in https://github.com/NewGraphEnvironment/dff-2022/ as per the readme at https://github.com/NewGraphEnvironment/dff-2022/tree/master/scripts/qgis.


system2was really tricky to try to run outside of theworkng directoryso we are going to try outprocessx`.


Let’s check the status of our project. First we make a function for our processx call.

library(processx)

# Define the command and working directory
command <- "conda"
args <- c('run', '-n', 'dff2', 'mergin', 'status')
working_directory <- "/Users/airvine/Projects/gis/py_client_error"

mergin_call <- function(){
  result <- tryCatch({
    run(
      command,
      args = args,
      echo = TRUE,            # Print the command output live
      wd = working_directory, # Set the working directory
      spinner = TRUE,         # Show a spinner
      timeout = 60            # Timeout after 60 seconds
    )
  }, error = function(e) {
    # Handle errors: e.g., print a custom error message
    cat("An error occurred: ", e$message, "\n")
    NULL  # Return NULL or another appropriate value
  })
  
  # Check if the command was successful
  if (!is.null(result)) {
    cat("Exit status:", result$status, "\n")
    cat("Output:\n", result$stdout)
  } else {
    cat("Failed to execute the command properly.\n")
  }
}
result <- mergin_call()
## -\|/-\|/-\|/-### Server changes:
## ### Local changes:
## ### Local changes summary ###
## 
## \ Exit status: 0 
## Output:
##  ### Server changes:
## ### Local changes:
## ### Local changes summary ###

Now let’s add a file to the project and check the status again

# Create a new file in the project directory
# to make reporoduable we will use a random number between 1:10000 appended

file_name <- paste0("/Users/airvine/Projects/gis/py_client_error/test_", sample(1:10000, 1), ".txt")
system2("touch", args = c(file_name))

result <- mergin_call()
## -\|/-\|/-\|/-### Server changes:
## ### Local changes:
## 
## >>> Added:
## + test_7736.txt
## ### Local changes summary ###
## 
## \| Exit status: 0 
## Output:
##  ### Server changes:
## ### Local changes:
## 
## >>> Added:
## + test_7736.txt
## ### Local changes summary ###

Now we make a change to form_pscis.gpkg programatically with R.

# sf::st_layers(paste0(working_directory, "/form_pscis.gpkg"))
form_in <- sf::st_read(paste0(working_directory, "/form_pscis.gpkg"))
## Reading layer `form_pscis' from data source 
##   `/Users/airvine/Projects/gis/py_client_error/form_pscis.gpkg' using driver `GPKG'
## Simple feature collection with 16 features and 88 fields
## Geometry type: POINT
## Dimension:     XY
## Bounding box:  xmin: 948064.9 ymin: 447409.8 xmax: 1036232 ymax: 1000702
## Projected CRS: NAD83 / BC Albers
# this is adding rows to the table 
form_in |> 
  dplyr::mutate(my_crossing_reference = sample(1:10000, 1)) |> 
  sf::st_write(paste0(working_directory, "/form_pscis.gpkg"), "form_pscis", append = FALSE)
## Deleting layer `form_pscis' using driver `GPKG'
## Writing layer `form_pscis' to data source 
##   `/Users/airvine/Projects/gis/py_client_error/form_pscis.gpkg' using driver `GPKG'
## Writing 16 features with 88 fields and geometry type Point.

Now we check the status again.

# Execute the command using processx::run within tryCatch
result <- mergin_call()
## -\|/-\|/-\|/-/var/folders/mg/h910y2c54fsc99qj74dyrjph0000gn/T/tmpvz_nbhcp: line 3: 93200 Segmentation fault: 11  mergin status
## 
## ERROR conda.cli.main_run:execute(124): `conda run mergin status` failed. (See above for error)
## \| An error occurred:  System command 'conda' failed 
## Failed to execute the command properly.

Really weird. This was all good the first time I ran it. Now it throws an error. We should look closer at the changes in that file.

Push Changes to Server

Let’s push the changes to the server. Doesn’t matter if it wouldn’t let us see the status….

client$push_project('/Users/airvine/Projects/gis/py_client_error')

# client$pull_project('/Users/airvine/Projects/gis/py_client_error')

Now we check the status again.

# Execute the command using processx::run within tryCatch
result <- mergin_call()
## -\|/-\|/-\|/-### Server changes:
## ### Local changes:
## ### Local changes summary ###
## 
## \| Exit status: 0 
## Output:
##  ### Server changes:
## ### Local changes:
## ### Local changes summary ###

That seems to clear everything up… Why? Why does it need to be pushed for the status to work? Guessing that it is because reading in the file and making changes in R alters the schema of the geopackage somehow and geodiff cannot properly track it so throws the segmentation error.


Let’s quickly visualize the differences in the file before and after our change. This is all within R - with the table as a sf object so is not telling us the difference in things like column types between QGIS and R.


Hmm. Let’s just test to see if even reading it and burning it back to file updated will cause the error. We don’t actually even read it into an sf object


sf::st_read(paste0(working_directory, "/form_pscis.gpkg")) |> 
  sf::st_write(paste0(working_directory, "/form_pscis.gpkg"), "form_pscis", update = TRUE)
## Reading layer `form_pscis' from data source 
##   `/Users/airvine/Projects/gis/py_client_error/form_pscis.gpkg' using driver `GPKG'
## Simple feature collection with 16 features and 88 fields
## Geometry type: POINT
## Dimension:     XY
## Bounding box:  xmin: 948064.9 ymin: 447409.8 xmax: 1036232 ymax: 1000702
## Projected CRS: NAD83 / BC Albers
## Updating layer `form_pscis' to data source `/Users/airvine/Projects/gis/py_client_error/form_pscis.gpkg' using driver `GPKG'
## Updating existing layer form_pscis
## Writing 16 features with 88 fields and geometry type Point.
result <- mergin_call()
## -\|/-\|/-\|/-/var/folders/mg/h910y2c54fsc99qj74dyrjph0000gn/T/tmpn3qeooc4: line 3: 93219 Segmentation fault: 11  mergin status
## 
## ERROR conda.cli.main_run:execute(124): `conda run mergin status` failed. (See above for error)
## \| An error occurred:  System command 'conda' failed 
## Failed to execute the command properly.

Ok. Looks like that is maybe the culprit. Let’s visualize the differences if we can.

Visualize Differences in geopackage file and sf object

Here is the gpkg in QGIS.

## {
##   "description":"/Users/airvine/Projects/gis/py_client_error/form_pscis.gpkg",
##   "driverShortName":"GPKG",
##   "driverLongName":"GeoPackage",
##   "layers":[
##     {
##       "name":"form_pscis",
##       "metadata":{},
##       "geometryFields":[
##         {
##           "name":"geom",
##           "type":"Point",
##           "nullable":true,
##           "extent":[
##             948064.874210938,
##             447409.81078381301,
##             1036232.08018285,
##             1000701.60915885
##           ],
##           "coordinateSystem":{
##             "wkt":"PROJCRS[\"NAD83 / BC Albers\",\n    BASEGEOGCRS[\"NAD83\",\n        DATUM[\"North American Datum 1983\",\n            ELLIPSOID[\"GRS 1980\",6378137,298.257222101,\n                LENGTHUNIT[\"metre\",1]]],\n        PRIMEM[\"Greenwich\",0,\n            ANGLEUNIT[\"degree\",0.0174532925199433]],\n        ID[\"EPSG\",4269]],\n    CONVERSION[\"British Columbia Albers\",\n        METHOD[\"Albers Equal Area\",\n            ID[\"EPSG\",9822]],\n        PARAMETER[\"Latitude of false origin\",45,\n            ANGLEUNIT[\"degree\",0.0174532925199433],\n            ID[\"EPSG\",8821]],\n        PARAMETER[\"Longitude of false origin\",-126,\n            ANGLEUNIT[\"degree\",0.0174532925199433],\n            ID[\"EPSG\",8822]],\n        PARAMETER[\"Latitude of 1st standard parallel\",50,\n            ANGLEUNIT[\"degree\",0.0174532925199433],\n            ID[\"EPSG\",8823]],\n        PARAMETER[\"Latitude of 2nd standard parallel\",58.5,\n            ANGLEUNIT[\"degree\",0.0174532925199433],\n            ID[\"EPSG\",8824]],\n        PARAMETER[\"Easting at false origin\",1000000,\n            LENGTHUNIT[\"metre\",1],\n            ID[\"EPSG\",8826]],\n        PARAMETER[\"Northing at false origin\",0,\n            LENGTHUNIT[\"metre\",1],\n            ID[\"EPSG\",8827]]],\n    CS[Cartesian,2],\n        AXIS[\"(E)\",east,\n            ORDER[1],\n            LENGTHUNIT[\"metre\",1]],\n        AXIS[\"(N)\",north,\n            ORDER[2],\n            LENGTHUNIT[\"metre\",1]],\n    USAGE[\n        SCOPE[\"Province-wide spatial data management.\"],\n        AREA[\"Canada - British Columbia.\"],\n        BBOX[48.25,-139.04,60.01,-114.08]],\n    ID[\"EPSG\",3005]]",
##             "projjson":{
##               "$schema":"https://proj.org/schemas/v0.7/projjson.schema.json",
##               "type":"ProjectedCRS",
##               "name":"NAD83 / BC Albers",
##               "base_crs":{
##                 "name":"NAD83",
##                 "datum":{
##                   "type":"GeodeticReferenceFrame",
##                   "name":"North American Datum 1983",
##                   "ellipsoid":{
##                     "name":"GRS 1980",
##                     "semi_major_axis":6378137,
##                     "inverse_flattening":298.257222101
##                   }
##                 },
##                 "coordinate_system":{
##                   "subtype":"ellipsoidal",
##                   "axis":[
##                     {
##                       "name":"Geodetic latitude",
##                       "abbreviation":"Lat",
##                       "direction":"north",
##                       "unit":"degree"
##                     },
##                     {
##                       "name":"Geodetic longitude",
##                       "abbreviation":"Lon",
##                       "direction":"east",
##                       "unit":"degree"
##                     }
##                   ]
##                 },
##                 "id":{
##                   "authority":"EPSG",
##                   "code":4269
##                 }
##               },
##               "conversion":{
##                 "name":"British Columbia Albers",
##                 "method":{
##                   "name":"Albers Equal Area",
##                   "id":{
##                     "authority":"EPSG",
##                     "code":9822
##                   }
##                 },
##                 "parameters":[
##                   {
##                     "name":"Latitude of false origin",
##                     "value":45,
##                     "unit":"degree",
##                     "id":{
##                       "authority":"EPSG",
##                       "code":8821
##                     }
##                   },
##                   {
##                     "name":"Longitude of false origin",
##                     "value":-126,
##                     "unit":"degree",
##                     "id":{
##                       "authority":"EPSG",
##                       "code":8822
##                     }
##                   },
##                   {
##                     "name":"Latitude of 1st standard parallel",
##                     "value":50,
##                     "unit":"degree",
##                     "id":{
##                       "authority":"EPSG",
##                       "code":8823
##                     }
##                   },
##                   {
##                     "name":"Latitude of 2nd standard parallel",
##                     "value":58.5,
##                     "unit":"degree",
##                     "id":{
##                       "authority":"EPSG",
##                       "code":8824
##                     }
##                   },
##                   {
##                     "name":"Easting at false origin",
##                     "value":1000000,
##                     "unit":"metre",
##                     "id":{
##                       "authority":"EPSG",
##                       "code":8826
##                     }
##                   },
##                   {
##                     "name":"Northing at false origin",
##                     "value":0,
##                     "unit":"metre",
##                     "id":{
##                       "authority":"EPSG",
##                       "code":8827
##                     }
##                   }
##                 ]
##               },
##               "coordinate_system":{
##                 "subtype":"Cartesian",
##                 "axis":[
##                   {
##                     "name":"Easting",
##                     "abbreviation":"E",
##                     "direction":"east",
##                     "unit":"metre"
##                   },
##                   {
##                     "name":"Northing",
##                     "abbreviation":"N",
##                     "direction":"north",
##                     "unit":"metre"
##                   }
##                 ]
##               },
##               "scope":"Province-wide spatial data management.",
##               "area":"Canada - British Columbia.",
##               "bbox":{
##                 "south_latitude":48.25,
##                 "west_longitude":-139.04,
##                 "north_latitude":60.01,
##                 "east_longitude":-114.08
##               },
##               "id":{
##                 "authority":"EPSG",
##                 "code":3005
##               }
##             },
##             "dataAxisToSRSAxisMapping":[
##               1,
##               2
##             ]
##           }
##         }
##       ],
##       "featureCount":32,
##       "fidColumnName":"fid",
##       "fields":[
##         {
##           "name":"date_time_start",
##           "type":"DateTime",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"pscis_crossing_id",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"my_crossing_reference",
##           "type":"Integer",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"crew_members",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"aggregated_crossings_id",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"camera_id",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"site_id",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"gps_id",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"moti_chris_culvert_id",
##           "type":"Integer",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"gps_waypoint_number",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"stream_name",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"road_name",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"road_km_mark",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"road_tenure",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"crossing_type",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"crossing_subtype",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"crossing_fix",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"diameter_or_span_meters",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"length_or_width_meters",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"assessment_comment",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"rowid",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"date",
##           "type":"Date",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"utm_zone",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"easting",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"northing",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"continuous_embeddedment_yes_no",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"average_depth_embededdment_meters",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"resemble_channel_yes_no",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"backwatered_yes_no",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"percentage_backwatered",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"fill_depth_meters",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"outlet_drop_meters",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"outlet_pool_depth_0_01m",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"inlet_drop_yes_no",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"culvert_slope_percent",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"downstream_channel_width_meters",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"stream_slope",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"beaver_activity_yes_no",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"fish_observed_yes_no",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"valley_fill",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"habitat_value",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"stream_width_ratio",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"culvert_length_score",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"embed_score",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"outlet_drop_score",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"culvert_slope_score",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"stream_width_ratio_score",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"final_score",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"barrier_result",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"recommended_diameter_or_span_meters",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"source",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"time_start",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"X.rowid",
##           "type":"Real",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"mergin_user",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"erosion_issues",
##           "type":"Integer",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"embankment_fill_issues",
##           "type":"Integer",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"blockage_issues",
##           "type":"Integer",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"condition_rank",
##           "type":"Integer",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"condition_notes",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"likelihood_flood_event_affecting_culvert",
##           "type":"Integer",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"consequence_flood_event_affecting_culvert",
##           "type":"Integer",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"climate_change_flood_risk",
##           "type":"Integer",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"vulnerability_rank",
##           "type":"Integer",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"climate_notes",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"traffic_volume",
##           "type":"Integer",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"community_access",
##           "type":"Integer",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"cost",
##           "type":"Integer",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"constructability",
##           "type":"Integer",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"fish_bearing",
##           "type":"Integer",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"environmental_impacts",
##           "type":"Integer",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"priority_rank",
##           "type":"Integer",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"overall_rank",
##           "type":"Integer",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"priority_notes",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"photo_road",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"photo_upstream",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"photo_downstream",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"photo_inlet",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"photo_barrel",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"photo_outlet",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"photo_condition",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"photo_embankment_fill",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"photo_blockage",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"photo_paper_card",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"photo_extra1",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"photo_extra2",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"photo_extra1_tag",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"photo_extra2_tag",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         },
##         {
##           "name":"link_method_phase1",
##           "type":"String",
##           "nullable":true,
##           "uniqueConstraint":false
##         }
##       ]
##     }
##   ],
##   "metadata":{},
##   "domains":{},
##   "relationships":{}
## }
name type
date_time_start DateTime
pscis_crossing_id Real
my_crossing_reference Integer
crew_members String
aggregated_crossings_id Real
camera_id String
site_id Real
gps_id String
moti_chris_culvert_id Integer
gps_waypoint_number String
stream_name String
road_name String
road_km_mark Real
road_tenure String
crossing_type String
crossing_subtype String
crossing_fix String
diameter_or_span_meters Real
length_or_width_meters Real
assessment_comment String
rowid Real
date Date
utm_zone Real
easting Real
northing Real
continuous_embeddedment_yes_no String
average_depth_embededdment_meters Real
resemble_channel_yes_no String
backwatered_yes_no String
percentage_backwatered Real
fill_depth_meters Real
outlet_drop_meters Real
outlet_pool_depth_0_01m Real
inlet_drop_yes_no String
culvert_slope_percent Real
downstream_channel_width_meters Real
stream_slope Real
beaver_activity_yes_no String
fish_observed_yes_no String
valley_fill String
habitat_value String
stream_width_ratio Real
culvert_length_score Real
embed_score Real
outlet_drop_score Real
culvert_slope_score Real
stream_width_ratio_score Real
final_score Real
barrier_result String
recommended_diameter_or_span_meters Real
source String
time_start String
X.rowid Real
mergin_user String
erosion_issues Integer
embankment_fill_issues Integer
blockage_issues Integer
condition_rank Integer
condition_notes String
likelihood_flood_event_affecting_culvert Integer
consequence_flood_event_affecting_culvert Integer
climate_change_flood_risk Integer
vulnerability_rank Integer
climate_notes String
traffic_volume Integer
community_access Integer
cost Integer
constructability Integer
fish_bearing Integer
environmental_impacts Integer
priority_rank Integer
overall_rank Integer
priority_notes String
photo_road String
photo_upstream String
photo_downstream String
photo_inlet String
photo_barrel String
photo_outlet String
photo_condition String
photo_embankment_fill String
photo_blockage String
photo_paper_card String
photo_extra1 String
photo_extra2 String
photo_extra1_tag String
photo_extra2_tag String
link_method_phase1 String

Here is gpkg columns and types in R

gpkp_r <- sf::st_read(paste0(working_directory, "/form_pscis.gpkg"))
## Reading layer `form_pscis' from data source 
##   `/Users/airvine/Projects/gis/py_client_error/form_pscis.gpkg' using driver `GPKG'
## Simple feature collection with 32 features and 88 fields
## Geometry type: POINT
## Dimension:     XY
## Bounding box:  xmin: 948064.9 ymin: 447409.8 xmax: 1036232 ymax: 1000702
## Projected CRS: NAD83 / BC Albers
# maek tibble of colun names and class
gpkp_r_sum <- purrr::map2_dfr(
  names(gpkp_r), 
  sapply(gpkp_r, function(col) class(col)[1]), 
  ~tibble::tibble(name = .x, type = .y)
)

fpr::fpr_kable(gpkp_r_sum, font = params$font_set)
name type
date_time_start POSIXct
pscis_crossing_id numeric
my_crossing_reference integer
crew_members character
aggregated_crossings_id numeric
camera_id character
site_id numeric
gps_id character
moti_chris_culvert_id integer
gps_waypoint_number character
stream_name character
road_name character
road_km_mark numeric
road_tenure character
crossing_type character
crossing_subtype character
crossing_fix character
diameter_or_span_meters numeric
length_or_width_meters numeric
assessment_comment character
rowid numeric
date Date
utm_zone numeric
easting numeric
northing numeric
continuous_embeddedment_yes_no character
average_depth_embededdment_meters numeric
resemble_channel_yes_no character
backwatered_yes_no character
percentage_backwatered numeric
fill_depth_meters numeric
outlet_drop_meters numeric
outlet_pool_depth_0_01m numeric
inlet_drop_yes_no character
culvert_slope_percent numeric
downstream_channel_width_meters numeric
stream_slope numeric
beaver_activity_yes_no character
fish_observed_yes_no character
valley_fill character
habitat_value character
stream_width_ratio numeric
culvert_length_score numeric
embed_score numeric
outlet_drop_score numeric
culvert_slope_score numeric
stream_width_ratio_score numeric
final_score numeric
barrier_result character
recommended_diameter_or_span_meters numeric
source character
time_start character
X.rowid numeric
mergin_user character
erosion_issues integer
embankment_fill_issues integer
blockage_issues integer
condition_rank integer
condition_notes character
likelihood_flood_event_affecting_culvert integer
consequence_flood_event_affecting_culvert integer
climate_change_flood_risk integer
vulnerability_rank integer
climate_notes character
traffic_volume integer
community_access integer
cost integer
constructability integer
fish_bearing integer
environmental_impacts integer
priority_rank integer
overall_rank integer
priority_notes character
photo_road character
photo_upstream character
photo_downstream character
photo_inlet character
photo_barrel character
photo_outlet character
photo_condition character
photo_embankment_fill character
photo_blockage character
photo_paper_card character
photo_extra1 character
photo_extra2 character
photo_extra1_tag character
photo_extra2_tag character
link_method_phase1 character
geom sfc_POINT

Let’s compare the differences:

identical(gpkp_q_sum, gpkp_r_sum)
## [1] FALSE
waldo::compare(gpkp_q_sum, gpkp_r_sum)
## `attr(old, 'row.names')[86:88]`: 86 87 88   
## `attr(new, 'row.names')[86:89]`: 86 87 88 89
## 
## old vs new
##                                                  name      type
## - old[1, ]  date_time_start                           DateTime 
## + new[1, ]  date_time_start                           POSIXct  
## - old[2, ]  pscis_crossing_id                         Real     
## + new[2, ]  pscis_crossing_id                         numeric  
## - old[3, ]  my_crossing_reference                     Integer  
## + new[3, ]  my_crossing_reference                     integer  
## - old[4, ]  crew_members                              String   
## + new[4, ]  crew_members                              character
## - old[5, ]  aggregated_crossings_id                   Real     
## + new[5, ]  aggregated_crossings_id                   numeric  
## - old[6, ]  camera_id                                 String   
## + new[6, ]  camera_id                                 character
## - old[7, ]  site_id                                   Real     
## + new[7, ]  site_id                                   numeric  
## - old[8, ]  gps_id                                    String   
## + new[8, ]  gps_id                                    character
## - old[9, ]  moti_chris_culvert_id                     Integer  
## + new[9, ]  moti_chris_culvert_id                     integer  
## - old[10, ] gps_waypoint_number                       String   
## + new[10, ] gps_waypoint_number                       character
## and 79 more ...
## 
## `old$name[86:88]`: "photo_extra1_tag" "photo_extra2_tag" "link_method_phase1"       
## `new$name[86:89]`: "photo_extra1_tag" "photo_extra2_tag" "link_method_phase1" "geom"
## 
##      old$type   | new$type                   
##  [1] "DateTime" - "POSIXct"   [1]            
##  [2] "Real"     - "numeric"   [2]            
##  [3] "Integer"  - "integer"   [3]            
##  [4] "String"   - "character" [4]            
##  [5] "Real"     - "numeric"   [5]            
##  [6] "String"   - "character" [6]            
##  [7] "Real"     - "numeric"   [7]            
##  [8] "String"   - "character" [8]            
##  [9] "Integer"  - "integer"   [9]            
## [10] "String"   - "character" [10]           
##  ... ...          ...         and 79 more ...

Summary

Ok. Seems perhaps reasonable that we get some discontinuity with modifications in R. We do have systems to track changes with git and csvs. Clunky but they do work somewhat (I think).

Modify geopackage table in QGIS by hand.

What about if we do something by hand? That should be fine and geodiff should be able to track.

First let’s remove the project again and redownload to confirm our status is all good

system2("rm", args = c("-rf", "/Users/airvine/Projects/gis/py_client_error"))

# now we download
system2('conda', args = c('run', '-n', 'dff2', 'mergin download newgraph/py_client_error /Users/airvine/Projects/gis/py_client_error'), stdout = TRUE)
## [1] "Downloading into /Users/airvine/Projects/gis/py_client_error"
## [2] ""                                                            
## [3] "Done"                                                        
## [4] ""
# check status
result <- mergin_call()
## -\|/-\|/-\|/-\### Server changes:
## ### Local changes:
## ### Local changes summary ###
## 
## |/ Exit status: 0 
## Output:
##  ### Server changes:
## ### Local changes:
## ### Local changes summary ###

We opened the project in QGIS and make a change to the form_pscis table “by hand”. See screenshot.

# append scripts/qgis/mergin/ to the front would make snese...
knitr::include_graphics("fig/Screen Shot 2024-05-02 at 10.02.21 AM.png")

Then we checked the status again.

result <- mergin_call()
## -\|/-\|/-\|/-### Server changes:
## ### Local changes:
## ### Local changes summary ###
## 
## \| Exit status: 0 
## Output:
##  ### Server changes:
## ### Local changes:
## ### Local changes summary ###

Ok. So I don’t think we have a problem with R and types. As interesting as all that is - it seems that even a manual change in the gpkg triggers an error. We will have to look into this further.

TO DO:

Test if altering the gpkg by hand throws error if the file was never read into R….. There could be corruptions in the file history over time that trigger error with geodiff every time there is a change…..

Test py-api-client v0.8.3

Let’s have a look at what happens if we try to check the status with the py-api-client v0.8.3 version.

# Define the command and working directory
args <- c('run', '-n', 'dff', 'mergin', 'status')

result <- mergin_call()
## -\|/-\|/-\|/-Error: HTTP Error: 405 METHOD NOT ALLOWED
## URL: https://app.merginmaps.com/v1/project/py_client_error?since=v23
## Method: GET
## Detail: The method is not allowed for the requested URL.
## 
## \| Exit status: 0 
## Output:
##  Error: HTTP Error: 405 METHOD NOT ALLOWED
## URL: https://app.merginmaps.com/v1/project/py_client_error?since=v23
## Method: GET
## Detail: The method is not allowed for the requested URL.
args <- c('run', '-n', 'dff', 'mergin', 'push')
result <- mergin_call()
## -\|/-\|/-\|/-Error: HTTP Error: 405 METHOD NOT ALLOWED
## URL: https://app.merginmaps.com/v1/project/py_client_error
## Method: GET
## Detail: The method is not allowed for the requested URL.
## 
## \| Exit status: 0 
## Output:
##  Error: HTTP Error: 405 METHOD NOT ALLOWED
## URL: https://app.merginmaps.com/v1/project/py_client_error
## Method: GET
## Detail: The method is not allowed for the requested URL.

We can’t push!!

Looks like things have improved! Nice work ninjas!! ⚔️

Download Project with QGIS plugin

Let’s try one last thing. We are going to download the file with the QGIS plugin, make a change to the form_pscis table and then check the status again.


First we remove the project.

# turned off - remove the project
system2("rm", args = c("-rf", "/Users/airvine/Projects/gis/py_client_error"))

Then we download the project with the QGIS plugin.

# Download the project with the QGIS plugin
# oddly seems we neeed to run from here vs with scripts/qgis/mergin/ at the front. don't get it
knitr::include_graphics("fig/Screen Shot 2024-05-02 at 12.42.11 AM.png")

Check the status with v0.9.0 again

args <- c('run', '-n', 'dff2', 'mergin', 'status')

result <- mergin_call()
## -\|/-\|/-\|/-\### Server changes:
## ### Local changes:
## ### Local changes summary ###
## 
## |/ Exit status: 0 
## Output:
##  ### Server changes:
## ### Local changes:
## ### Local changes summary ###

Now we make a trivial change with R - same as before.

form_in <- sf::st_read(paste0(working_directory, "/form_pscis.gpkg"))
## Reading layer `form_pscis' from data source 
##   `/Users/airvine/Projects/gis/py_client_error/form_pscis.gpkg' using driver `GPKG'
## Simple feature collection with 16 features and 88 fields
## Geometry type: POINT
## Dimension:     XY
## Bounding box:  xmin: 948064.9 ymin: 447409.8 xmax: 1036232 ymax: 1000702
## Projected CRS: NAD83 / BC Albers
form_in |> 
  dplyr::mutate(my_crossing_reference = sample(1:10000, 1)) |> 
  sf::st_write(paste0(working_directory, "/form_pscis.gpkg"), delete_dsn = TRUE)
## Deleting source `/Users/airvine/Projects/gis/py_client_error/form_pscis.gpkg' using driver `GPKG'
## Writing layer `form_pscis' to data source 
##   `/Users/airvine/Projects/gis/py_client_error/form_pscis.gpkg' using driver `GPKG'
## Writing 16 features with 88 fields and geometry type Point.

Check the status again.

args <- c('run', '-n', 'dff2', 'mergin', 'status')
result <- mergin_call()
## -\|/-\|/-\|/-/var/folders/mg/h910y2c54fsc99qj74dyrjph0000gn/T/tmpoa97nbya: line 3: 93289 Segmentation fault: 11  mergin status
## 
## ERROR conda.cli.main_run:execute(124): `conda run mergin status` failed. (See above for error)
## \| An error occurred:  System command 'conda' failed 
## Failed to execute the command properly.

This is the captured output from the terminal. There is more with processx but that is not relevant I don’t think.

/var/folders/mg/h910y2c54fsc99qj74dyrjph0000gn/T/tmp6qsmgvr7: line 3: 82287 Segmentation fault: 11  mergin status

ERROR conda.cli.main_run:execute(124): `conda run mergin status` failed. (See above for error)

AHA!! It seemed like it was the QGIS plugin. But our results were inconsistent. Crazy. Not sure what is going on.

Push Changes to Server

This is interesting too though.. We can still push and pull no problem.

args <- c('run', '-n', 'dff2', 'mergin', 'push')

result <- run(
  command,
  args = args,
  echo = TRUE,            # Print the command output live (similar to stdout = TRUE in system2)
  wd = working_directory, # Set the working directory
  spinner = TRUE,         # Show a spinner in the R console while the command runs
  timeout = 60            # Set a timeout for the command to complete
)

Let’s get the chunk ready to do our commits of these files to address our issue. We turn it off though.

# we need to derive the properpath to the file for git to see it
path_file_out <- paste0(
    tools::file_path_sans_ext(params$file_in),
  '.html'
)

gert::git_add(
  c(params$file_in, path_file_out)
)

message <- "initial commit file to address #166"
message <- "update file to address #166 and include git commits"

gert::git_commit(message)

gert::git_push()

We can read our docs online so lets add a comment in the issue that points to the file:

# construct the url of the viewer
url_viewer <- paste0(
  "http://htmlpreview.github.io/?",
  params$repo_url,
  "blob/",
  params$repo_branch,
  "/",
  path_file_out
)

gh::gh(endpoint = "POST /repos/{owner}/{repo}/issues/{issue_number}/comments",
       body = paste0("The documentation for this is available here ", url_viewer, " ."),
       owner = params$repo_owner,
       repo = params$repo_name,
       issue_number = 166
)
## {
##   "url": "https://api.github.com/repos/NewGraphEnvironment/dff-2022/issues/comments/2091156922",
##   "html_url": "https://github.com/NewGraphEnvironment/dff-2022/issues/166#issuecomment-2091156922",
##   "issue_url": "https://api.github.com/repos/NewGraphEnvironment/dff-2022/issues/166",
##   "id": 2091156922,
##   "node_id": "IC_kwDOHyEXG858pIW6",
##   "user": {
##     "login": "NewGraphEnvironment",
##     "id": 46538103,
##     "node_id": "MDQ6VXNlcjQ2NTM4MTAz",
##     "avatar_url": "https://avatars.githubusercontent.com/u/46538103?u=aaf04ae3fc7d68a0dec3202562cedcf27ce188a8&v=4",
##     "gravatar_id": "",
##     "url": "https://api.github.com/users/NewGraphEnvironment",
##     "html_url": "https://github.com/NewGraphEnvironment",
##     "followers_url": "https://api.github.com/users/NewGraphEnvironment/followers",
##     "following_url": "https://api.github.com/users/NewGraphEnvironment/following{/other_user}",
##     "gists_url": "https://api.github.com/users/NewGraphEnvironment/gists{/gist_id}",
##     "starred_url": "https://api.github.com/users/NewGraphEnvironment/starred{/owner}{/repo}",
##     "subscriptions_url": "https://api.github.com/users/NewGraphEnvironment/subscriptions",
##     "organizations_url": "https://api.github.com/users/NewGraphEnvironment/orgs",
##     "repos_url": "https://api.github.com/users/NewGraphEnvironment/repos",
##     "events_url": "https://api.github.com/users/NewGraphEnvironment/events{/privacy}",
##     "received_events_url": "https://api.github.com/users/NewGraphEnvironment/received_events",
##     "type": "User",
##     "site_admin": false
##   },
##   "created_at": "2024-05-02T17:45:36Z",
##   "updated_at": "2024-05-02T17:45:36Z",
##   "author_association": "OWNER",
##   "body": "The documentation for this is available here http://htmlpreview.github.io/?https://github.com/NewGraphEnvironment/dff-2022/blob/master/scripts/qgis/mergin/py_client_error.html .",
##   "reactions": {
##     "url": "https://api.github.com/repos/NewGraphEnvironment/dff-2022/issues/comments/2091156922/reactions",
##     "total_count": 0,
##     "+1": 0,
##     "-1": 0,
##     "laugh": 0,
##     "hooray": 0,
##     "confused": 0,
##     "heart": 0,
##     "rocket": 0,
##     "eyes": 0
##   },
##   "performed_via_github_app": {}
## }
rmarkdown::render(input = params$file_in, output_file = params$file_out)